# -*- coding: utf-8 -*-
import scrapy, json
from scrapy.conf import settings
from mySpider.items import UmengItems


class UmengAppChannelsSpider(scrapy.Spider):
    name = 'umeng'
    allowed_domains = ['mobile.umeng.com']
    start_urls = ['http://mobile.umeng.com/apps']
    _cookie = settings['UM_COOKIE']
    _headers = settings['_HEADERS']

    def start_requests(self):
        yield scrapy.Request(self.start_urls[0], headers = self._headers, cookies = self._cookie, callback = self.appid)

    def appid(self, response):
        _li = response.xpath('//*[@id="doc"]/div[1]/div/div/div[1]/ul/li')
        for index, l in enumerate(_li):
            _appid = l.xpath('@app_id').extract()
            _package = l.xpath('text()').extract()
            if index == 0:
                continue
            else:
                yield scrapy.Request(
                    'http://mobile.umeng.com/apps/%s/channels/load_chart_data?start_date=2017-10-16&end_date=2017-11-07&time_unit=daily&stats=installation&' % _appid[0],
                    headers = self._headers,
                    cookies = self._cookie,
                    callback = self._result,
                    meta = {'package': _package[1].strip().replace('_', '.')}
                )

    def _result(self, response):
        _res_text = response.text
        _res_json = json.loads(_res_text)
        item = UmengItems()
        for i, date in enumerate(_res_json['dates']):
            for x, stats in enumerate(_res_json['stats']):
                item['package'] = response.meta['package']
                item['date'] = date
                item['channel'] = stats['name']
                item['quantity'] = int(stats['data'][i])
                yield item